# coding=utf-8
"""
    @project: 15python_spider
    @Author：frank
    @file： 07_xicidaili_proxypool.py
    @date：2024/2/20 19:24
"""
import requests
from lxml import etree
from fake_useragent import UserAgent

url = 'https://www.kuaidaili.com/free/fps/1/'


# 获取User-Agent
def get_random_ua():
    ua = UserAgent()
    return ua.random


# 获取所有代理IP的地址和端口号
# [{'http':'http://xxxx:xx'}]

def get_ip_list():
    headers = {'User-Agent': get_random_ua()}
    html = requests.get(url, headers=headers, verify=False).text
    # 解析
    parse_html = etree.HTML(html)
    r_list = parse_html.xpath("//tr")
    # 空列表，存放所有ip和port
    proxy_list = []
    for r in r_list[1:]:
        ip = r.xpath('./td[1]/text()')[0]
        port = r.xpath('./td[2]/text()')[0]
        proxy_list.append({'http': 'http://{}:{}'.format(ip, port),
                           'https': 'https://{}:{}'.format(ip, port)
                           })

    return proxy_list


def proxy_pool():
    proxy_list = get_ip_list()
    # 可用代理ip列表
    useful_proxy = []
    # 测试代理是否可用
    for proxy in proxy_list:
        headers = {'User-Agent': get_random_ua()}
        try:
            res = requests.get(url='http://httpbin.org/get', headers=headers, proxies=proxy, timeout=5)
            print(res.text)
            print(proxy)
            useful_proxy.append(proxy)
        except Exception as e:
            print('{}不能用'.format(proxy))
            continue


if __name__ == "__main__":
    proxy_pool()
